package learning.data;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import learning.data.document.TreeDocument;
import learning.data.document.TreeDocument.ExampleClass;
import learning.util.SparseVector;

public class TreeDataReader {

	static String file = "O:/unix/projects/pardosa/data10/raphaelh/learning/birth_date";
	
	public static Dataset<TreeDocument> read(String file) throws IOException {
		
		// right now, all labels and features get thrown together - no matter which
		// class they are from. we'll have to check, but i think the algorithms
		// would prefer them being bucketed seperately
		
		Dataset<TreeDocument> dataset;
		{
			HashMap<String, Integer> featureIds;
			String[] featureNames;			
			HashMap<String, Integer> labelIds;
			String[] labelNames;
			List<TreeDocument> docs;
			
			{
				// read labels
				labelIds = new HashMap<String, Integer>();
				BufferedReader br = new BufferedReader(new InputStreamReader
						(new FileInputStream(file + "." + "labels"), "utf-8"));
				String line = null;
				while ((line = br.readLine()) != null) {
					labelIds.put(line.substring(line.indexOf("\t") + 1), 
							Integer.parseInt(line.substring(0, line.indexOf("\t"))));				
				}
				labelNames = new String[labelIds.size()];
				for (Map.Entry<String, Integer> e : labelIds.entrySet())
					labelNames[e.getValue()] = e.getKey();
			}
			{
				// read features
				featureIds = new HashMap<String, Integer>();
				BufferedReader br = new BufferedReader(new InputStreamReader
						(new FileInputStream(file + "." + "features"), "utf-8"));
				String line = null;
				while ((line = br.readLine()) != null) {
					featureIds.put(line.substring(line.indexOf("\t") + 1), 
							Integer.parseInt(line.substring(0, line.indexOf("\t"))));
				}
				featureNames = new String[featureIds.size()];
				for (Map.Entry<String, Integer> e : featureIds.entrySet())
					featureNames[e.getValue()] = e.getKey();				
			}
			{
				// read dataset
				docs = new ArrayList<TreeDocument>();
				BufferedReader br = new BufferedReader(new InputStreamReader
						(new FileInputStream(file + "." + "data"), "utf-8"));
				
				List<ExampleClass> exampleClasses = new ArrayList<ExampleClass>();
				String currentClass = null;
				List<SparseVector> features = new ArrayList<SparseVector>();
				List<String> tokens = new ArrayList<String>();
				List<int[]> spans = new ArrayList<int[]>();
				List<Integer> labels = new ArrayList<Integer>();
				
				String line = null;
				while ((line = br.readLine()) != null) {
					if (line.length() == 0) {
						TreeDocument doc = new TreeDocument(tokens.toArray(new String[0]),
								exampleClasses.toArray(new ExampleClass[0]));
						docs.add(doc);
						continue;
					}
					String[] cols = line.split(" ");
					
					if (currentClass == null || !currentClass.equals(cols[0])) {
						if (currentClass != null) {
							exampleClasses.add(new ExampleClass(currentClass,
									to2dIntArray(spans), features.toArray(new SparseVector[0]),
									toIntArray(labels), tokens.toArray(new String[0])));
							features.clear();
							spans.clear();
							labels.clear();
						}
						currentClass = cols[0];
					}
					
					tokens.add(cols[1]);
					labels.add(labelIds.get(cols[cols.length-1]));
					
					int[] fids = new int[cols.length-2];
					float[] fvals = new float[cols.length-2];
					for (int i=0; i < fids.length; i++) {
						fids[i] = featureIds.get(cols[i+1]);
						fvals[i] = 1.0f;
					}					
					features.add(new SparseVector(fids, fvals, fids.length));
				}
			}
			dataset = new Dataset<TreeDocument>(docs.toArray(new TreeDocument[0]),
					featureNames, featureIds, labelNames, labelIds);
		}
		return dataset;
	}
		
	private static int[] toIntArray(List<Integer> list) {
		int[] ia = new int[list.size()];
		for (int i = 0; i < list.size(); i++) ia[i] = list.get(i);
		return ia;
	}
	
	private static int[][] to2dIntArray(List<int[]> list) {
		int[][] ia = new int[list.size()][2];
		for (int i = 0; i < list.size(); i++)
			System.arraycopy(list.get(i), 0, ia[i], 0, 2);
		return ia;
	}
}
